package io.renren.modules.job.task.crawl;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;

/**
 * 定时爬取新闻
 *
 * @auther: Freeman Liu
 * @date: 2020/11/4 14:37
 */
@Component
public class CrawlNews {

    @Autowired
    private NewsPipeline newsPipeline;
    @Autowired
    private NewsProcessor newsProcessor;

    String startUrl = "http://www.cnncw.cn/pdlm/lypd/bendiyou/";

    /**
     * 每天4点爬取新闻
     */
//    @Scheduled(cron = "0 0 4 * * ?")
    public void crawlNews() {
        Spider.create(newsProcessor)
                .addUrl(startUrl)
//                .addPipeline(new ConsolePipeline())
                .addPipeline(newsPipeline)
//                .runAsync();  //异步爬取
                .run();
    }
}
